{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-0a12e15fc6da>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From c:\\users\\yaorui_01\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From c:\\users\\yaorui_01\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./MNIST_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From c:\\users\\yaorui_01\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From c:\\users\\yaorui_01\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ./MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting ./MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From c:\\users\\yaorui_01\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"./MNIST_data\",one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化权重，定义一个维度为shape，均值为0，标准差默认为0.1的数组\n",
    "def initialize(shape, stddev=0.1):\n",
    "    return tf.truncated_normal(shape, stddev=stddev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(\"float\", [None, 784])   # 定义输入，batch的大小暂时没定，784是特征数，这里指一个图片的像素个数，28x28\n",
    "y = tf.placeholder(\"int64\", [None,10])  #定义输出，batch的大小暂时没定\n",
    "learning_rate = tf.placeholder(\"float\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow里没有专门的swish函数，这里我们自定义一个swish激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def swish(x):\n",
    "    return x*tf.nn.sigmoid(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activation(x):\n",
    "    #return tf.nn.selu(x)\n",
    "    #return tf.nn.relu(x)\n",
    "    #return tf.nn.sigmoid(x)\n",
    "    return swish(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 这里使用3层神经网络，前两层每层神经元的个数增加到500，每一层的权重初始化使用MSRA初始化，标准差的选择和每层输入的个数有关"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设定第一层神经网络的个数\n",
    "L1_units_count = 500\n",
    "\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count],stddev=0.003))   # 初始化第一层神经网络的权重\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))  # 初始化第一层神经网络的偏置\n",
    "logits_1 = tf.matmul(x, W_1) + b_1  # 输入与权重的矩阵相差再加上偏置\n",
    "output_1 = activation(logits_1) # 把这一层的输出经过激活在输出到下一层\n",
    "\n",
    "L2_units_count = 500 \n",
    "W_2 = tf.Variable(initialize([L1_units_count, L2_units_count],stddev=0.006))\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "output_2 = activation(logits_2)\n",
    "\n",
    "L3_units_count = 10 # 第三层神经网络的个数\n",
    "W_3 = tf.Variable(initialize([L2_units_count, L3_units_count],stddev=0.006))\n",
    "b_3 = tf.Variable(initialize([L3_units_count]))\n",
    "logits_3 = tf.matmul(output_2, W_3) + b_3  \n",
    "\n",
    "logits = logits_3 # 最终输出的原始数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 定义学习器和损失函数，这里增加了L2损失，使用l2_loss来计算每一层权重的l2损失，每一层的l2损失需要加起来，总体损失中，L2损失前所乘以的学习率需要足够小，不然模型完全不收敛"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=logits))\n",
    "l2_loss = tf.nn.l2_loss(W_1) + tf.nn.l2_loss(W_2)+ tf.nn.l2_loss(W_3)\n",
    "total_loss = cross_entropy + 7e-5*l2_loss # 交叉熵损失加上学习率乘以l2损失\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 定义正确率的计算方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = tf.nn.softmax(logits) #经过softmax得到最终的概率分布\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(pred, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 执行训练，学习率随着训练次数的增加逐步减小"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 1.134339, l2_loss: 32.367203, total loss: 1.136605\n",
      "0.73\n",
      "0.6892\n",
      "step 200, entropy loss: 0.577859, l2_loss: 83.374023, total loss: 0.583695\n",
      "0.92\n",
      "0.8684\n",
      "step 300, entropy loss: 0.262209, l2_loss: 106.970772, total loss: 0.269697\n",
      "0.95\n",
      "0.9204\n",
      "step 400, entropy loss: 0.187388, l2_loss: 127.804153, total loss: 0.196334\n",
      "0.98\n",
      "0.9243\n",
      "step 500, entropy loss: 0.363531, l2_loss: 145.596863, total loss: 0.373723\n",
      "0.94\n",
      "0.9374\n",
      "step 600, entropy loss: 0.101484, l2_loss: 162.557953, total loss: 0.112863\n",
      "0.99\n",
      "0.9468\n",
      "step 700, entropy loss: 0.230112, l2_loss: 175.805847, total loss: 0.242419\n",
      "0.98\n",
      "0.9473\n",
      "step 800, entropy loss: 0.108869, l2_loss: 186.286865, total loss: 0.121909\n",
      "1.0\n",
      "0.955\n",
      "step 900, entropy loss: 0.144179, l2_loss: 197.337341, total loss: 0.157993\n",
      "1.0\n",
      "0.9611\n",
      "step 1000, entropy loss: 0.036207, l2_loss: 206.212738, total loss: 0.050642\n",
      "1.0\n",
      "0.9648\n",
      "step 1100, entropy loss: 0.128400, l2_loss: 217.879349, total loss: 0.143652\n",
      "0.99\n",
      "0.9653\n",
      "step 1200, entropy loss: 0.038529, l2_loss: 228.556778, total loss: 0.054528\n",
      "1.0\n",
      "0.9679\n",
      "step 1300, entropy loss: 0.082040, l2_loss: 236.797241, total loss: 0.098616\n",
      "1.0\n",
      "0.9692\n",
      "step 1400, entropy loss: 0.083802, l2_loss: 245.039917, total loss: 0.100955\n",
      "0.99\n",
      "0.9607\n",
      "step 1500, entropy loss: 0.091335, l2_loss: 251.963760, total loss: 0.108972\n",
      "1.0\n",
      "0.9727\n",
      "step 1600, entropy loss: 0.190890, l2_loss: 259.473724, total loss: 0.209053\n",
      "0.98\n",
      "0.9558\n",
      "step 1700, entropy loss: 0.056959, l2_loss: 267.702484, total loss: 0.075698\n",
      "1.0\n",
      "0.9721\n",
      "step 1800, entropy loss: 0.051592, l2_loss: 275.905151, total loss: 0.070905\n",
      "1.0\n",
      "0.9724\n",
      "step 1900, entropy loss: 0.038239, l2_loss: 276.340302, total loss: 0.057583\n",
      "1.0\n",
      "0.9791\n",
      "step 2000, entropy loss: 0.010515, l2_loss: 276.483185, total loss: 0.029869\n",
      "1.0\n",
      "0.9785\n",
      "step 2100, entropy loss: 0.065854, l2_loss: 277.047089, total loss: 0.085247\n",
      "0.99\n",
      "0.9789\n",
      "step 2200, entropy loss: 0.109699, l2_loss: 277.442871, total loss: 0.129120\n",
      "0.99\n",
      "0.9807\n",
      "step 2300, entropy loss: 0.022514, l2_loss: 278.085938, total loss: 0.041980\n",
      "1.0\n",
      "0.9816\n",
      "step 2400, entropy loss: 0.019415, l2_loss: 278.892517, total loss: 0.038938\n",
      "1.0\n",
      "0.9812\n",
      "step 2500, entropy loss: 0.012107, l2_loss: 279.848206, total loss: 0.031696\n",
      "1.0\n",
      "0.9803\n",
      "step 2600, entropy loss: 0.007641, l2_loss: 279.877594, total loss: 0.027233\n",
      "1.0\n",
      "0.9803\n",
      "step 2700, entropy loss: 0.032777, l2_loss: 279.928711, total loss: 0.052372\n",
      "0.99\n",
      "0.9806\n",
      "step 2800, entropy loss: 0.017716, l2_loss: 279.952118, total loss: 0.037313\n",
      "0.99\n",
      "0.9807\n",
      "step 2900, entropy loss: 0.087067, l2_loss: 280.011993, total loss: 0.106668\n",
      "0.97\n",
      "0.9809\n",
      "step 3000, entropy loss: 0.034302, l2_loss: 280.034332, total loss: 0.053905\n",
      "0.99\n",
      "0.9812\n",
      "step 3100, entropy loss: 0.010707, l2_loss: 280.086060, total loss: 0.030313\n",
      "1.0\n",
      "0.9814\n",
      "step 3200, entropy loss: 0.013833, l2_loss: 280.149353, total loss: 0.033443\n",
      "1.0\n",
      "0.9813\n",
      "step 3300, entropy loss: 0.020402, l2_loss: 280.240234, total loss: 0.040019\n",
      "0.99\n",
      "0.9814\n",
      "step 3400, entropy loss: 0.023709, l2_loss: 280.288513, total loss: 0.043329\n",
      "0.99\n",
      "0.9813\n",
      "step 3500, entropy loss: 0.022606, l2_loss: 280.321594, total loss: 0.042229\n",
      "1.0\n",
      "0.9816\n",
      "step 3600, entropy loss: 0.093876, l2_loss: 280.401123, total loss: 0.113504\n",
      "0.97\n",
      "0.9816\n",
      "step 3700, entropy loss: 0.010019, l2_loss: 280.455658, total loss: 0.029651\n",
      "1.0\n",
      "0.9814\n",
      "step 3800, entropy loss: 0.027004, l2_loss: 280.502014, total loss: 0.046639\n",
      "1.0\n",
      "0.9817\n",
      "step 3900, entropy loss: 0.009596, l2_loss: 280.571930, total loss: 0.029236\n",
      "1.0\n",
      "0.9817\n",
      "step 4000, entropy loss: 0.053521, l2_loss: 280.646820, total loss: 0.073166\n",
      "0.98\n",
      "0.9817\n",
      "step 4100, entropy loss: 0.049601, l2_loss: 280.697083, total loss: 0.069249\n",
      "0.99\n",
      "0.9818\n",
      "step 4200, entropy loss: 0.022142, l2_loss: 280.754333, total loss: 0.041795\n",
      "0.99\n",
      "0.9818\n",
      "step 4300, entropy loss: 0.042590, l2_loss: 280.818359, total loss: 0.062247\n",
      "0.99\n",
      "0.9821\n",
      "step 4400, entropy loss: 0.014017, l2_loss: 280.858673, total loss: 0.033677\n",
      "1.0\n",
      "0.9817\n",
      "step 4500, entropy loss: 0.018449, l2_loss: 280.917328, total loss: 0.038113\n",
      "1.0\n",
      "0.9811\n",
      "step 4600, entropy loss: 0.011487, l2_loss: 280.983917, total loss: 0.031156\n",
      "1.0\n",
      "0.9815\n",
      "step 4700, entropy loss: 0.062258, l2_loss: 281.039001, total loss: 0.081931\n",
      "0.99\n",
      "0.9817\n",
      "step 4800, entropy loss: 0.076778, l2_loss: 281.078217, total loss: 0.096453\n",
      "0.97\n",
      "0.982\n",
      "step 4900, entropy loss: 0.037454, l2_loss: 281.140198, total loss: 0.057134\n",
      "0.99\n",
      "0.9819\n",
      "step 5000, entropy loss: 0.067622, l2_loss: 281.205017, total loss: 0.087307\n",
      "0.98\n",
      "0.9816\n",
      "step 5100, entropy loss: 0.005534, l2_loss: 281.212036, total loss: 0.025219\n",
      "1.0\n",
      "0.9817\n",
      "step 5200, entropy loss: 0.070163, l2_loss: 281.219452, total loss: 0.089849\n",
      "0.98\n",
      "0.9817\n",
      "step 5300, entropy loss: 0.009721, l2_loss: 281.224823, total loss: 0.029407\n",
      "1.0\n",
      "0.9817\n",
      "step 5400, entropy loss: 0.010729, l2_loss: 281.226593, total loss: 0.030415\n",
      "1.0\n",
      "0.9819\n",
      "step 5500, entropy loss: 0.021337, l2_loss: 281.231354, total loss: 0.041023\n",
      "1.0\n",
      "0.9817\n",
      "step 5600, entropy loss: 0.011392, l2_loss: 281.236084, total loss: 0.031079\n",
      "1.0\n",
      "0.9817\n",
      "step 5700, entropy loss: 0.039086, l2_loss: 281.243042, total loss: 0.058773\n",
      "0.99\n",
      "0.9818\n",
      "step 5800, entropy loss: 0.042905, l2_loss: 281.247223, total loss: 0.062592\n",
      "0.97\n",
      "0.9818\n",
      "step 5900, entropy loss: 0.016757, l2_loss: 281.254303, total loss: 0.036445\n",
      "1.0\n",
      "0.9816\n",
      "step 6000, entropy loss: 0.098217, l2_loss: 281.258789, total loss: 0.117905\n",
      "0.99\n",
      "0.9818\n",
      "step 6100, entropy loss: 0.049522, l2_loss: 281.264404, total loss: 0.069211\n",
      "0.99\n",
      "0.9818\n",
      "step 6200, entropy loss: 0.011609, l2_loss: 281.269196, total loss: 0.031298\n",
      "1.0\n",
      "0.9819\n",
      "step 6300, entropy loss: 0.025942, l2_loss: 281.273926, total loss: 0.045631\n",
      "0.99\n",
      "0.982\n",
      "step 6400, entropy loss: 0.020950, l2_loss: 281.282043, total loss: 0.040640\n",
      "1.0\n",
      "0.9819\n",
      "step 6500, entropy loss: 0.012622, l2_loss: 281.287354, total loss: 0.032312\n",
      "1.0\n",
      "0.9819\n",
      "step 6600, entropy loss: 0.045656, l2_loss: 281.293396, total loss: 0.065346\n",
      "0.98\n",
      "0.9819\n",
      "step 6700, entropy loss: 0.027855, l2_loss: 281.296509, total loss: 0.047545\n",
      "0.99\n",
      "0.9818\n",
      "step 6800, entropy loss: 0.006916, l2_loss: 281.304047, total loss: 0.026607\n",
      "1.0\n",
      "0.9819\n",
      "step 6900, entropy loss: 0.021502, l2_loss: 281.311768, total loss: 0.041193\n",
      "1.0\n",
      "0.9819\n",
      "step 7000, entropy loss: 0.057631, l2_loss: 281.316071, total loss: 0.077323\n",
      "0.98\n",
      "0.9819\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "for step in range(7000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    if step<1800:\n",
    "        lr = 1.2\n",
    "    elif step<2500:\n",
    "        lr = 0.3\n",
    "    elif step<5000:\n",
    "        lr = 0.03\n",
    "    else:\n",
    "        lr=0.003\n",
    "    _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [optimizer, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y: batch_ys, learning_rate:lr})\n",
    "  \n",
    "    if (step+1) % 100 == 0:\n",
    "        print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "        # Test trained model\n",
    "        print(sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys}))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 增加了神经元个数每层500，并增加了一个隐层，使用swish激活函数，batch_size为100，经过7000步的训练，达到了在测试集上98%的正确率。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 这里在训练了2500步左右后，正确率始终维持98%出头"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
